{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-7fb5eb44687f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtfds\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeras\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlayers\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mlayers\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\ProgramData\\Anaconda\\envs\\keras\\lib\\site-packages\\tensorflow_datasets\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     45\u001b[0m \u001b[1;31m# needs to happen before anything else, since the imports below will try to\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     46\u001b[0m \u001b[1;31m# import tensorflow, too.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 47\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtf_compat\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     48\u001b[0m \u001b[0mtf_compat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mensure_tf_install\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     49\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\ProgramData\\Anaconda\\envs\\keras\\lib\\site-packages\\tensorflow_datasets\\core\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     26\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconstants\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0madd_data_dir\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 28\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdataset_builder\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mBeamBasedBuilder\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     29\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdataset_builder\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mBuilderConfig\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     30\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdataset_builder\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mDatasetBuilder\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\ProgramData\\Anaconda\\envs\\keras\\lib\\site-packages\\tensorflow_datasets\\core\\dataset_builder.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     34\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mapi_utils\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mconstants\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mdownload\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mlazy_imports_lib\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnaming\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\ProgramData\\Anaconda\\envs\\keras\\lib\\site-packages\\tensorflow_datasets\\core\\download\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchecksums\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0madd_checksums_dir\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 20\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload_manager\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mDownloadConfig\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     21\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload_manager\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mDownloadManager\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownloader\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mDownloadError\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\ProgramData\\Anaconda\\envs\\keras\\lib\\site-packages\\tensorflow_datasets\\core\\download\\download_manager.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     33\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mutils\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     34\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mchecksums\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 35\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mdownloader\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     36\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mextractor\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     37\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorflow_datasets\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mresource\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mresource_lib\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\ProgramData\\Anaconda\\envs\\keras\\lib\\site-packages\\tensorflow_datasets\\core\\download\\downloader.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     32\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpromise\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     33\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mrequests\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 34\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mrequests\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mutils\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mextract_zipped_paths\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     35\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmoves\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0murllib\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mv2\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mImportError\u001b[0m: cannot import name 'extract_zipped_paths'"
     ],
     "ename": "ImportError",
     "evalue": "cannot import name 'extract_zipped_paths'",
     "output_type": "error"
    }
   ],
   "source": [
    "import tensorflow_datasets as tfds\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras.layers as layers\n",
    "\n",
    "import time\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True,\n",
    "                              as_supervised=True)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "train_examples,val_examples = examples['train'],examples['validation']\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "tokenizer_en = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n",
    "    (en.numpy() for pt,en in train_examples),target_vocab_size=10000\n",
    ")\n",
    "tokenizer_pt = tfds.features.text.SubwordTextEncoder.build_from_corpus(\n",
    "    (pt.numpy() for pt,en in train_examples),train_examples=10000\n",
    ")\n",
    "\n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "sample_str = 'hello word, tensorflow 2'\n",
    "tokenized_str = tokenizer_en.encode(sample_str)\n",
    "print(tokenized_str)\n",
    "original_str = tokenizer_en.decode(tokenized_str)\n",
    "print(original_str)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def encode(lang1,lang2):\n",
    "    lang1 = [tokenizer_pt.vocab_size] + tokenizer_pt.encode(\n",
    "        lang1.numpy() + [tokenizer_pt.vocab_size + 1]\n",
    "    )\n",
    "    lang2 = [tokenizer_en.vocab_size] + tokenizer_en.encode(\n",
    "        lang2.numpy()) + [tokenizer_en.vocab_size+1]\n",
    "    return lang1,lang2\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "MAX_LENGTH = 40\n",
    "def filter_long_sent(x,y,max_length=MAX_LENGTH):\n",
    "    return tf.logical_and(tf.size(x) <= max_length,\n",
    "                          tf.size(y) <= max_length)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def tf_encode(pt,en):\n",
    "    return tf.py_function(encode,[pt,en],[tf.int64,tf.int64])\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "BUFFER_SIZE = 2000\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "train_dataset = train_examples.map(tf_encode)\n",
    "train_dataset = train_dataset.filter(filter_long_sent)\n",
    "train_dataset = train_dataset.cache()\n",
    "train_dataset = train_dataset.padded_batch(\n",
    "    BATCH_SIZE,padded_shapes=([40],[40])\n",
    ")\n",
    "train_dataset =  train_dataset.prefetch(tf.data.experimental.AUTOTUNE)\n",
    "\n",
    "val_dataset = val_examples.map(tf_encode)\n",
    "val_dataset = val_dataset.filter(filter_long_sent).padded_batch(\n",
    "    BATCH_SIZE,padded_shapes=([40],[40])\n",
    ")\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "de_batch,en_batch = next(iter(train_dataset))\n",
    "de_batch,en_batch"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [],
   "source": [
    "def get_angles(pos, i, d_model):\n",
    "    # 这里的i等价与上面公式中的2i和2i+1\n",
    "    angle_rates = 1 / np.power(10000, (2*(i // 2))/ np.float32(d_model))\n",
    "    return pos * angle_rates"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [],
   "source": [
    "def positional_encoding(position, d_model):\n",
    "    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n",
    "                           np.arange(d_model)[np.newaxis,:],\n",
    "                           d_model)\n",
    "    # 第2i项使用sin\n",
    "    sines = np.sin(angle_rads[:, 0::2])\n",
    "    # 第2i+1项使用cos\n",
    "    cones = np.cos(angle_rads[:, 1::2])\n",
    "    pos_encoding = np.concatenate([sines, cones], axis=-1)\n",
    "    pos_encoding = pos_encoding[np.newaxis, ...]\n",
    "    \n",
    "    return tf.cast(pos_encoding, dtype=tf.float32)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-1bdf29570ca9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mpos_encoding\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpositional_encoding\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m50\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m512\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpos_encoding\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpcolormesh\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpos_encoding\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcmap\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'RdBu'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'positional_encoding' is not defined"
     ],
     "ename": "NameError",
     "evalue": "name 'positional_encoding' is not defined",
     "output_type": "error"
    }
   ],
   "source": [
    "pos_encoding = positional_encoding(50,512)\n",
    "print(pos_encoding.shape)\n",
    "\n",
    "plt.pcolormesh(pos_encoding[0], cmap='RdBu')\n",
    "plt.xlabel('Depth')\n",
    "plt.xlim((0, 512))\n",
    "plt.ylabel('Position')\n",
    "plt.colorbar()\n",
    "plt.show() "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "<tf.Tensor: shape=(2, 1, 1, 5), dtype=float32, numpy=\narray([[[[0., 0., 1., 1., 0.]]],\n\n\n       [[[0., 0., 0., 1., 1.]]]], dtype=float32)>"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 4
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "def create_padding_mask(seq):\n",
    "    seq = tf.cast(tf.math.equal(seq,0),tf.float32)\n",
    "    return seq[:,np.newaxis,np.newaxis,:]\n",
    "\n",
    "create_padding_mask([[1,2,0,0,3],[3,4,5,0,0]])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "tf.Tensor(\n",
      "[[0. 1. 1.]\n",
      " [0. 0. 1.]\n",
      " [0. 0. 0.]], shape=(3, 3), dtype=float32)\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "def create_look_ahead_mark(size):\n",
    "    mark = 1 - tf.linalg.band_part(tf.ones((size,size)),-1,0)\n",
    "    return mark\n",
    "tmp = create_look_ahead_mark(3)\n",
    "print(tmp)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [],
   "source": [
    "def scaled_dot_product_attention(q,k,v,mask):\n",
    "    matmul_qk = tf.matmul(q,k,transpose_b=True)\n",
    "    dk = tf.cast(tf.shape(k)[-1],tf.float32)\n",
    "    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
    "    \n",
    "    if mask is not None:\n",
    "        scaled_attention_logits += (mask * -1e9)\n",
    "    \n",
    "    attention_weights = tf.nn.softmax(scaled_attention_logits,axis=-1)\n",
    "    output = tf.matmul(attention_weights,v)\n",
    "    return output,attention_weights\n",
    "    "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [],
   "source": [
    "def print_out(q,k,v):\n",
    "    temp_out,temp_att = scaled_dot_product_attention(\n",
    "        q,k,v,None\n",
    "    )\n",
    "    print(\"attention weight:\")\n",
    "    print(temp_att)\n",
    "    print(\"output:\")\n",
    "    print(temp_out)\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "attention weight:\n",
      "tf.Tensor([[0. 1. 0. 0.]], shape=(1, 4), dtype=float32)\n",
      "output:\n",
      "tf.Tensor([[10.  0.]], shape=(1, 2), dtype=float32)\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "np.set_printoptions(suppress=True)\n",
    "temp_k = tf.constant([\n",
    "    [10,0,0],\n",
    "    [0,10,0],\n",
    "    [0,0,10],\n",
    "    [0,0,10]\n",
    "],dtype=tf.float32)\n",
    "\n",
    "temp_v = tf.constant([\n",
    "    [1,0],\n",
    "    [10,0],\n",
    "    [100,5],\n",
    "    [1000,6]\n",
    "],dtype=tf.float32)\n",
    "\n",
    "temp_q = tf.constant([[0,10,0]],dtype=tf.float32)\n",
    "print_out(temp_q,temp_k,temp_v)\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "attention weight:\n",
      "tf.Tensor([[0.  0.  0.5 0.5]], shape=(1, 4), dtype=float32)\n",
      "output:\n",
      "tf.Tensor([[550.    5.5]], shape=(1, 2), dtype=float32)\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "temp_q = tf.constant([[0,0,10]],dtype=tf.float32)\n",
    "print_out(temp_q,temp_k,temp_v)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "attention weight:\n",
      "tf.Tensor([[0.5 0.5 0.  0. ]], shape=(1, 4), dtype=float32)\n",
      "output:\n",
      "tf.Tensor([[5.5 0. ]], shape=(1, 2), dtype=float32)\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "temp_q = tf.constant([[10,10,0]], dtype=tf.float32)\n",
    "print_out(temp_q, temp_k, temp_v)\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "attention weight:\n",
      "tf.Tensor(\n",
      "[[0.  0.  0.5 0.5]\n",
      " [0.  1.  0.  0. ]\n",
      " [0.5 0.5 0.  0. ]], shape=(3, 4), dtype=float32)\n",
      "output:\n",
      "tf.Tensor(\n",
      "[[550.    5.5]\n",
      " [ 10.    0. ]\n",
      " [  5.5   0. ]], shape=(3, 2), dtype=float32)\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "# 依次放入每个query\n",
    "temp_q = tf.constant([[0, 0, 10], [0, 10, 0], [10, 10, 0]], dtype=tf.float32)  # (3, 3)\n",
    "print_out(temp_q, temp_k, temp_v)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "outputs": [],
   "source": [
    "class MutilHeadAttention(tf.keras.layers.Layer):\n",
    "    def __init__(self,d_model,num_heads):\n",
    "        super(MutilHeadAttention,self).__init__()\n",
    "        self.num_heads = num_heads\n",
    "        self.d_model = d_model\n",
    "        \n",
    "        assert d_model % num_heads == 0\n",
    "        \n",
    "        self.depth = d_model // num_heads\n",
    "        self.wq = tf.keras.layers.Dense(d_model)\n",
    "        self.wk = tf.keras.layers.Dense(d_model)\n",
    "        self.wv = tf.keras.layers.Dense(d_model)\n",
    "        self.dense = tf.keras.layers.Dense(d_model)\n",
    "    \n",
    "    def split_heads(self,x,batch_size):\n",
    "        x = tf.reshape(x,(batch_size,-1,self.num_heads,self.depth))\n",
    "        return tf.transpose(x,perm=[0,2,1,3])\n",
    "    \n",
    "    def call(self,v,k,q,mask):\n",
    "        batch_size = tf.shape(q)[0]\n",
    "        \n",
    "        q = self.wq(q)\n",
    "        k = self.wk(k)\n",
    "        v = self.wv(v)\n",
    "        \n",
    "        q = self.split_heads(q,batch_size)\n",
    "        k = self.split_heads(k,batch_size)\n",
    "        v = self.split_heads(v,batch_size)\n",
    "        \n",
    "        scaled_attention,attention_weights = scaled_dot_product_attention(\n",
    "            q,k,v,mask\n",
    "        )\n",
    "        \n",
    "        scaled_attention = tf.transpose(scaled_attention, [0, 2, 1, 3]) \n",
    "        concat_attention = tf.reshape(scaled_attention, \n",
    "                                      (batch_size, -1, self.d_model))\n",
    "        output = self.dense(concat_attention)\n",
    "        return output, attention_weights\n",
    "        "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "(1, 60, 512) (1, 8, 60, 60)\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "temp_mha = MutilHeadAttention(d_model=512,num_heads=8)\n",
    "y = tf.random.uniform((1,60,512))\n",
    "output,att = temp_mha(y,k=y,q=y,mask=None)\n",
    "print(output.shape,att.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "data": {
      "text/plain": "TensorShape([64, 50, 512])"
     },
     "metadata": {},
     "output_type": "execute_result",
     "execution_count": 34
    }
   ],
   "source": [
    "def point_wise_feed_forward_network(d_model,diff):\n",
    "    return tf.keras.Sequential([\n",
    "        tf.keras.layers.Dense(diff,activation='relu'),\n",
    "        tf.keras.layers.Dense(d_model)\n",
    "    ])\n",
    "\n",
    "sample_fnn = point_wise_feed_forward_network(512,2048)\n",
    "sample_fnn(tf.random.uniform((64,50,512))).shape\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "outputs": [],
   "source": [
    "class LayerNormalization(tf.keras.layers.Layer):\n",
    "    def __init__(self,epsilon=1e-6,**kwargs):\n",
    "        self.eps = epsilon\n",
    "        super(LayerNormalization,self).__init__(**kwargs)\n",
    "    \n",
    "    def build(self,input_shape):\n",
    "        self.gamma = self.add_weight(name='gamma',shape=input_shape[-1:],\n",
    "                                     initializer=tf.ones_initializer(),trainable=True)\n",
    "        self.beta = self.add_weight(name='beta',shape=input_shape[-1:],\n",
    "                                    initializer=tf.zeros_initializer(),trainable=True)\n",
    "        super(LayerNormalization,self).build(input_shape)\n",
    "    \n",
    "    def call(self,x):\n",
    "        mean = tf.keras.backend.mean(x,axis=-1,keepdims=True)\n",
    "        std = tf.keras.backend.std(x,axis=-1,keepdims=True)\n",
    "        return self.gamma * (x - mean) /(std + self.eps) + self.beta\n",
    "    \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return input_shape\n",
    "    \n",
    "    \n",
    "        \n",
    "    "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "outputs": [],
   "source": [
    "class EncoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self,d_model,n_heads,ddf,dropout_rate=0.1):\n",
    "        super(EncoderLayer,self).__init__()\n",
    "        \n",
    "        self.mha = MutilHeadAttention(d_model,n_heads)\n",
    "        self.ffn = point_wise_feed_forward_network(d_model,ddf)\n",
    "        \n",
    "        self.layernorm1 = LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = LayerNormalization(epsilon=1e-6)\n",
    "        \n",
    "        self.dropout1 = tf.keras.layers.Dropout(dropout_rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(dropout_rate)\n",
    "    \n",
    "    def call(self,inputs,training,mask):\n",
    "        att_output,_ = self.mha(inputs,inputs,inputs,mask)\n",
    "        att_output = self.dropout1(att_output,training=training)\n",
    "        out1 = self.layernorm1(inputs + att_output)\n",
    "        \n",
    "        ffn_output = self.ffn(out1)\n",
    "        ffn_output = self.dropout2(ffn_output,training=training)\n",
    "        out2 = self.layernorm2(out1 + ffn_output)\n",
    "        return out2\n",
    "        \n",
    "        "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "(64, 43, 512)\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "sample_encoder_layer = EncoderLayer(512,8,2048)\n",
    "sample_encoder_layer_output = sample_encoder_layer(\n",
    "    tf.random.uniform((64,43,512)),False,None\n",
    ")\n",
    "print(sample_encoder_layer_output.shape)\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "outputs": [],
   "source": [
    "class DecoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self,d_model,num_heads,dff,drop_rate=0.1):\n",
    "        super(DecoderLayer,self).__init__()\n",
    "        \n",
    "        self.mha1 = MutilHeadAttention(d_model,num_heads)\n",
    "        self.mha2 = MutilHeadAttention(d_model,num_heads)\n",
    "        \n",
    "        self.ffn = point_wise_feed_forward_network(d_model,dff)\n",
    "        \n",
    "        self.layernorm1 = LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm3 = LayerNormalization(epsilon=1e-6)\n",
    "        \n",
    "        self.dropout1 = tf.keras.layers.Dropout(drop_rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(drop_rate)\n",
    "        self.dropout3 = tf.keras.layers.Dropout(drop_rate)\n",
    "    \n",
    "    def call(self,inputs,encode_out,training,\n",
    "             look_ahead_mask,padding_mask):\n",
    "        att1,att_weight1 = self.mha1(inputs,inputs,inputs,look_ahead_mask)\n",
    "        att1 = self.dropout1(att1,training=training)\n",
    "        out1 = self.layernorm1(inputs + att1)\n",
    "        \n",
    "        att2,att_weight2 = self.mha2(encode_out,encode_out,inputs,padding_mask)\n",
    "        att2 = self.dropout2(att2,training=training)\n",
    "        out2 = self.layernorm2(out1 + att2)\n",
    "        \n",
    "        ffn_out = self.ffn(out2)\n",
    "        ffn_out = self.dropout3(ffn_out,training=training)\n",
    "        out3 = self.layernorm3(out2+ ffn_out)\n",
    "        \n",
    "        return out3,att_weight1,att_weight2\n",
    "        \n",
    "        \n",
    "        \n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "(64, 50, 512)\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "sample_decoder_layer = DecoderLayer(512,8,2048)\n",
    "sample_decoder_layer_output,_,_ = sample_decoder_layer(\n",
    "    tf.random.uniform((64,50,512)),sample_encoder_layer_output,\n",
    "    False,None,None\n",
    ")\n",
    "print(sample_decoder_layer_output.shape)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.layers.Layer):\n",
    "    def __init__(self,n_layers,d_model,n_heads,ddf,\n",
    "                 input_vocab_size,max_seq_len,drop_rate=0.1):\n",
    "        super(Encoder,self).__init__()\n",
    "        self.n_layers = n_layers\n",
    "        self.d_model = d_model\n",
    "        \n",
    "        self.embedding = tf.keras.layers.Embedding(input_vocab_size,d_model)\n",
    "        self.pos_embedding = positional_encoding(max_seq_len,d_model)\n",
    "        \n",
    "        self.encode_layer = [EncoderLayer(d_model,n_heads,ddf,drop_rate)\n",
    "                             for _ in range(n_layers)]\n",
    "        \n",
    "        self.dropout = tf.keras.layers.Dropout(drop_rate)\n",
    "    \n",
    "    def call(self,inputs,training,mark):\n",
    "        seq_len = inputs.shape[1]\n",
    "        word_emb = self.embedding(inputs)\n",
    "        word_emb *= tf.math.sqrt(tf.cast(self.d_model,tf.float32))\n",
    "        emb = word_emb + self.pos_embedding[:,:seq_len,:]\n",
    "        x = self.dropout(emb,training=training)\n",
    "        for i in range(self.n_layers):\n",
    "            x = self.encode_layer[i](x,training,mark)\n",
    "        return x\n",
    "    \n",
    "        \n",
    "        \n",
    "        "
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "(64, 43, 512)\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "sample_encoder = Encoder(2,512,8,1024,5000,200)\n",
    "sample_encoder_output = sample_encoder(tf.random.uniform((64,120)),False,None)\n",
    "print(sample_encoder_layer_output.shape)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.layers.Layer):\n",
    "    def __init__(self,n_layers,d_model,n_heads,ddf,\n",
    "                 target_vocab_size,max_seq_len,drop_rate=0.1):\n",
    "        super(Decoder,self).__init__()\n",
    "        \n",
    "        self.d_model = d_model\n",
    "        self.n_layers = n_layers\n",
    "        \n",
    "        self.embedding = layers.Embedding(target_vocab_size,d_model)\n",
    "        self.pos_embedding = positional_encoding(max_seq_len,d_model)\n",
    "        \n",
    "        self.decoder_layers = [DecoderLayer(d_model,n_heads,ddf,drop_rate) \n",
    "                               for _ in range(n_layers)]\n",
    "        self.dropout = layers.Dropout(drop_rate)\n",
    "    \n",
    "    def call(self,inputs,encoder_out,training,\n",
    "             look_ahead_mark,padding_work):\n",
    "        seq_len = tf.shape(inputs)[1]\n",
    "        attention_weights = {}\n",
    "        h = self.embedding(inputs)\n",
    "        h *= tf.math.sqrt(tf.cast(self.d_model,tf.float32))\n",
    "        h += self.pos_embedding[:,:seq_len,:]\n",
    "        \n",
    "        h = self.dropout(h,training=training)\n",
    "        \n",
    "        for i in range(self.n_layers):\n",
    "            h,att_w1,att_w2 = self.decoder_layers[i](h,encoder_out,\n",
    "                                                     training,look_ahead_mark,\n",
    "                                                     padding_work)\n",
    "            attention_weights['decoder_layer{}_att_w1'.format(i+1)] = att_w1\n",
    "            attention_weights['decoder_layer{}_att_w2'.format(i+1)] = att_w2\n",
    "        \n",
    "        return h,attention_weights\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [
    {
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-46-6f1a6ca88fda>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0msample_decoder\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mDecoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m512\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m8\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1024\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m5000\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m200\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m ssample_decoder_output,attn = sample_decoder(tf.random.uniform((64,100)),\n\u001b[0;32m      4\u001b[0m                                              sample_encoder_output,False,None,None)\n\u001b[0;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mssample_decoder_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-45-14bdc9505b62>\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, n_layers, d_model, n_heads, ddf, target_vocab_size, max_seq_len, drop_rate)\u001b[0m\n\u001b[0;32m      8\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn_layers\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mn_layers\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0membedding\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlayers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mEmbedding\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget_vocab_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0md_model\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     11\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpos_embedding\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpositional_encoding\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmax_seq_len\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0md_model\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'layers' is not defined"
     ],
     "ename": "NameError",
     "evalue": "name 'layers' is not defined",
     "output_type": "error"
    }
   ],
   "source": [
    "sample_decoder = Decoder(2,512,8,1024,5000,200)\n",
    "ssample_decoder_output,attn = sample_decoder(tf.random.uniform((64,100)),\n",
    "                                             sample_encoder_output,False,None,None)\n",
    "print(ssample_decoder_output.shape)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "conda-env-keras-py",
   "language": "python",
   "display_name": "Python [conda env:keras]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}